* This code outputs the return and inflation expectations used in the main paper.
* It also produces Figure A.2 in the main paper.

********************************************************************************
* CONSTRUCT RETURN EXPECTATIONS
********************************************************************************
drop _all
set more off
set type double, permanently

local dir `c(pwd)'

*------------------------------------------------------------------------------*
* UBS/Gallup Survey                                                                            
*------------------------------------------------------------------------------*
* https://ropercenter.cornell.edu/ubs-index-investor-optimism-1996-2007
* Use the SAS program "import_ubs.sas" first to generate the txt file
insheet using "../Data/ubs.txt", clear

* Note: UBS/Gallup requires at least $10000 in selfdirected stocks/bonds/MF
replace futretown = . if (futretown > 95 | futretown < -95)
replace futretmkt = . if (futretmkt > 95 | futretmkt < -95)
replace ownpast = . if (ownpast < -100)
replace futretown = futretown/100
replace futretmkt = futretmkt/100
replace ownpast = ownpast/100

gen income = 15000 
replace income = 25000 if d9 == 4 
replace income = 35000 if d9 == 5
replace income = 45000 if d9 == 6
replace income = 55000 if d9 == 7
replace income = 67500 if d9 == 8
replace income = 87500 if d9 == 9
replace income = 200000 if d9 == 10
replace income = . if d9 > 10 

gen savings = 50000
replace savings = 150000 if d11 == 1
replace savings = 350000 if d11 == 2
replace savings = 750000 if d11 == 3
replace savings = 2000000 if d11 == 4 
replace savings = . if d11 == 5 

gen stryyyym = string(year)+"m"+string(month)
gen ymdate = monthly(stryyyym, "ym")
format ymdate %tm

drop s5 d9 d11 
saveold ubsmicro, version(12) replace 

collapse (mean) futretown futretmkt [aweight = wtfctr], by(ymdate year month)  

sort ymdate 
save basefile, replace 
 
*------------------------------------------------------------------------------*
* Michigan Survey                                                                     
*------------------------------------------------------------------------------*
* https://data.sca.isr.umich.edu/sda-public/cgi-bin/hsda2?setupfile=harcsda&datasetname=sca&ui=2&action=subset
* Select all variables
insheet using "../Data/MSC2020.csv", clear names
destring, replace

* Note: agg time-series data from MSC website calculates pstk_mean_all only from stockholders 
* see their chartbook, notes to Table 20 

replace pstk = . if pstk > 100 

gen month = yyyymm - yyyy*100
gen stryyyym = string(yyyy)+"m"+string(month)
gen ymdate = monthly(stryyyym, "ym")
drop yyyy month stryyyym

gen pstkinv = .
replace pstkinv = pstk if (invest == 1 & invamt >= 10000)

collapse (mean) pstk pstkinv [aweight = wt], by(ymdate) 

sort ymdate
merge ymdate using basefile
drop _merge
sort ymdate
save basefile, replace 
 
*------------------------------------------------------------------------------*
* Roper Survey                                                                                   
*------------------------------------------------------------------------------*

* Source: Roper Report 77-7, August 1977 
* Interview Dates: July 9, 1977 - July 16, 1977  less...
* Methodology: Conducted by Roper Organization, July 9 - July 16, 1977 and based on 2000 interviews. Sample: National adult. [USROPER.77-7.R25]	
* CSV format: month year increase same decrease undecided Source

insheet using "../Data/Roper.csv", clear names

gen incrmdecr = (increase-decrease)    //  
gen stryyyym = string(year)+"m"+string(month)
gen ymdate = monthly(stryyyym, "ym")

drop year month stryyyym

sort ymdate
merge ymdate using basefile
drop _merge 
sort ymdate


save basefile, replace 

*------------------------------------------------------------------------------*
* Conference Board Survey                                                                       
*------------------------------------------------------------------------------*
* https://www.conference-board.org/data/consumerconfidence.cfm
insheet using "../Data/ConferenceBoardStockExp.csv", clear names

gen stryyyym = string(year)+"m"+string(month)
gen ymdate = monthly(stryyyym, "ym")
format ymdate %tm

collapse (mean) increase same decrease, by(ymdate)
tsset ymdate

gen incrmdecr = increase-decrease

sort ymdate
merge ymdate using basefile
drop _merge 
sort ymdate
save basefile, replace 


*------------------------------------------------------------------------------*
* CRSP Dividend Yield
*------------------------------------------------------------------------------*
* Download the CRSP-->Stock/Security Files-->Stock Market Indexes at monthly frequency as a Stata data file
use "../Data/CRSPIndex.dta", clear

* Lagged month before survey data 
gen yyyy = year(DATE)
gen month = month(DATE)

gen mdiv = (vwretd-vwretx)*totval[_n-1]
gen stryyyym = string(yyyy)+"m"+string(month)

gen ymdate = monthly(stryyyym, "ym")
tsset ymdate 

gen crspdiv = L1.mdiv+L2.mdiv+L3.mdiv+L4.mdiv+L5.mdiv+L6.mdiv+L7.mdiv+L8.mdiv+L9.mdiv+L10.mdiv+L11.mdiv+L12.mdiv
gen dy = crspdiv/totval[_n-1]

keep ymdate yyyy month dy

merge ymdate using basefile
drop _merge 
sort ymdate
save basefile, replace 

*------------------------------------------------------------------------------*
* Treasury Yield 
*------------------------------------------------------------------------------*
* 1-year constant maturity yield from FRED 
* https://fred.stlouisfed.org/series/GS1
insheet using "../Data/GS1.csv", clear    //  Fred1yTreasConstMat.csv", clear

qui gen treas1y = GS1[_n-1]/100     // so that end of previous month yield associated with survey during the month
qui gen sdate = date(DATE, "YMD")

gen year = year(sdate)
gen month = month(sdate) 
gen stryyyyq = string(year)+"m"+string(month)
gen ymdate = monthly(stryyyyq, "ym")

keep ymdate treas1y

sort ymdate
merge ymdate using basefile
drop _merge 
sort ymdate
save basefile, replace 

*------------------------------------------------------------------------------*
* Additional Datapoints
*------------------------------------------------------------------------------* 
* Data point from Ameriks et al VRI survey August 2014 (based on http://ebp-projects.isr.umich.edu/VRI/surveys/VRI_survey3.pdf)
* growth in DJIA 
* (mean pstk = 0.51)
* mean growth = 0.06 
replace futretmkt = 0.06+dy if (yyyy == 2014 & month == 8) 

* Data point from Lewellen, Lease, Schlarbaum (1977): 
* survey mailed in summer 1972 
* Table 4 pre-tax expected returns (on their own portfolio) 
* Investor Age Estimate*
* (Years) 0-5 6-10 11-15 Over 15  Average estimate    Weight from Table 1
* Under 45 .......... .10 .35 .35 .20 11.3   15
* 45-54 ............. .11 .47 .25 .17 10.3     29 
* 55-64 ............. .23 .55 .14 .08 8.1     26
* 65 and over ........ .29 .54 .11 .06 7.4     30

* Lewellen, Lease, Schlarbaum (1974JF) report 9.3% overall average from the same data set
* They further report that they sent a survey to control group (fn. 3) that was similar except 
* a bit younger than the study group. This survey was sent in mid-1973 and the distribution of 
* attainable returns was "indistiguishable" from that in the study group. 

* This article also clarifies that they don't directly ask about return investors think 
* they can achieve themselves but rather something more like average return: 
* "What annual per-
* centage rate of return, before taxes, do you think is attainable on a regular
* basis from investments in common stocks, with moderate skill and understand-
* ing on the investor's part?" 
 
replace futretmkt = 0.093 if (yyyy == 1972 & month == 7) 
replace futretmkt = 0.093 if (yyyy == 1973 & month == 7) 

save basefile, replace 
 
*------------------------------------------------------------------------------*
* Splice Series
*------------------------------------------------------------------------------*
* Gallup (+other %exp): project market return expectations on own portfolio return expectations
reg futretmkt futretown 
predict gfutretmkthat, xb 
gen gfutretmkt = futretmkt
replace gfutretmkt = gfutretmkthat if gfutretmkt == . 
replace futretmkt = gfutretmkt if futretmkt == . 

* Project on MSC prob(up)  
* NOTE: for MSC prob that diversified MF increases in value: not entirely clear whether dividends 
*   included or not, but since it refers to value increase of an investment in the MF, probably yes
* "Suppose that tomorrow someone were to invest one thousand dollars in such a mutual fund. Please think about how much money this investment would be worth one year from
* now. What do you think is the percent chance that this one-thousand-dollar investment will increase in 
* value in the year ahead, so that it is worth more than one thousand dollars one year from now?" 
reg futretmkt pstkinv 
predict mfutretmkt, xb 
replace futretmkt = mfutretmkt if futretmkt == .

* Project on Conference board and Roper %up - %down
* Roper and conference board asks about stock "prices" so not clear whether dividends are included 
* seems questionable that individuals take into account changes in DY
* also not clear that "same" category is symmetric around zero price change 
* %increase then better than %increase-%decrease
gen y = futretmkt-dy
replace same = same+undecided if undecided ~= .  
gen x = increase/(decrease+same) 
reg y x 

predict py, xb 
gen cfutretmkt = py+dy
drop py
replace futretmkt = cfutretmkt if futretmkt == . 

gen eqpremtreas = futretmkt - treas1y

drop gfutretmkthat 

format ymdate %tm
 
keep if yyyy > 1971

saveold "`outdir'tsfile.dta", version(12) replace

drop if source == "ABC" 

*------------------------------------------------------------------------------* 
* Produce Figure A.2
*------------------------------------------------------------------------------*
twoway tsline gfutretmkt if (yyyy > 1996 & yyyy < 2010) , clcolor(blue) clpattern(solid) clwidth(medthick)  || ///
       tsline gfutretmkt if yyyy < 1997 , ms(o) mc(black) mlwidth(thin) msize(large) recast(scatter) || ///
       tsline gfutretmkt if yyyy > 2010 , ms(d) mc(black) mlwidth(thin) msize(large) recast(scatter) || ///
	   tsline mfutretmkt , clcolor(red) clpattern(solid) clwidth(medthick) || ///
	   tsline cfutretmkt if yyyy < 1987, ms(s) mc(black) mlwidth(thin) msize(large) recast(scatter) || ///
	   tsline cfutretmkt if yyyy > 1986, clcolor(green) clpattern(solid) clwidth(medthick) || , ///
	   xtitle("Month") ytitle("Expected Returns") ///
       legend(label(1 "UBS-Gallup") label(2 "LLS JF") label(3 "VRI") label(4 "Michigan") label(5 "Roper") label(6 "Conference Board")cols(2))  ///
       graphregion(color(white))
graph export "`outdir'ExpRets1.pdf", replace

*------------------------------------------------------------------------------* 
* Output Return Expectations
*------------------------------------------------------------------------------*
gen yqdate = qofd(dofm(ymdate)) 
format yqdate %tq
replace treas1y = . if futretmkt == .   // to avoid look-ahead bias when only early month in qtr available 
collapse (mean) futretmkt treas1y eqpremtreas, by(yqdate)  
gen yyyy = year(dofq(yqdate))
gen qtr =  quarter(dofq(yqdate))

keep yyyy qtr treas1y futretmkt eqpremtreas

export delimited using stockretexp, replace


********************************************************************************
* CONSTRUCT INFLATION EXPECTATIONS
********************************************************************************
*------------------------------------------------------------------------------*
* Constant Gain Learning Program
*------------------------------------------------------------------------------*

// 8 modified for seasonal AR(1) as in FOMC paper 

capture program drop constgain
program constgain
   version 10.1
   syntax anything
   tempname phit phit1 phit2 phit3 phit4 pf xf gcoh infl linfl l3infl llinfl ao lao dinfl dlinfl ageqtr ///
            c cc x xcurr xcurrlong y R smean smeanx perr1 perr2 perr3 perr4 g infl4 linfl4 l5infl l4infl l4infl ac winfl4 wlinfl4 th winfl wlinfl
   scalar `g' = `anything'
 
   matrix inflfullexp = J((2019-$datast)*4,1,-999) 
   matrix acfull = J((2019-$datast)*4,1,-999)
   matrix mafull = J((2019-$datast)*4,1,-999)
   
   local yrst = $yrst
   forvalues yr = `yrst'/2019 {
      forvalues q = 1/4 {
          local yri = (`yr'-$datast-1)*4+`q'      // yrinfl starts in qtr. $datast+1:01
          scalar `infl' = $pred[`yri',1]
          
          //initialize learning process
          
             if (`yr' < `yrst'+2) {     // 2 year lag needed to fill all the lags 
             
                if $gain < 4 {                       // simple weighted average                       
                   matrix `R' = 1
                   //matrix `x1' = 1   
                   matrix `phit' = `infl'
                   matrix `cc' = 0
                   matrix `x' = 1
                   //matrix `x3' = 1
                   //matrix `x2' = 1 
                   scalar `smean' = `infl'
                   scalar `smeanx' = 0
                }
                
                else if $gain == 4 {                          // Recursive least squares: AR(1)
                   //matrix `R' = (1, `infl'\ `infl', `infl'^2)  // dispersion in fitted exp looks identical with this one
                   matrix `R' = (1, 0\ 0, 0)
                   //matrix `x1' = (1,`infl')   
                   matrix `phit' = `infl'
                   matrix `cc' = (`phit' \ 0)     // dispersion in fitted exp looks identical with 0, 0.99 as starting value
                   matrix `x' = (1,0)
                   //matrix `x3' = (1,0)
                   //matrix `x2' = (1,0)      
                   scalar `smean' = `infl'
                   scalar `smeanx' = 0
                }
                
                else if $gain == 5 {                          // Stochastic gradient: AR(1)
                   matrix `R' = 1
                   //matrix `x1' = (1,`infl')   
                   matrix `phit' = `infl'
                   matrix `cc' = (`phit' \ 0)
                   matrix `x' = (1,0)
                   //matrix `x3' = (1,0)
                   //matrix `x2' = (1,0) 
                   scalar `smean' = `infl'
                   scalar `smeanx' = 0
                }
                
                else if $gain == 6 {                          // IMA(1,1) 
                   matrix `R' = (1, 0\ 0, 1)   // do not impose zero intercept in est., only in forecasting
                   matrix `phit' = `infl'
                   matrix `cc' =  (0 \ 0)
                   matrix `x' = (0, 0)
                   scalar `smean' = `infl'
                   scalar `smeanx' = 0
                   matrix `perr1' = 0
                   matrix `perr2' = 0
                   matrix `perr3' = 0
                   matrix `perr4' = 0
                }
                
                else if $gain == 7 {                          // Recursive least squares: ARMA(1,1) 
                   matrix `R' = (1, 0, 0 \ 0, 1, 0 \ 0, 0, 1)
                   matrix `phit' = `infl'
                   matrix `cc' = (0 \ 1 \ 0)     
                   matrix `x' = (1,0, 0)
                   scalar `smean' = `infl'
                   scalar `smeanx' = 0
                   matrix `perr1' = 0
                   matrix `perr2' = 0
                   matrix `perr3' = 0
                   matrix `perr4' = 0
                }
                
                else if $gain == 8 {                          // Seasonal AR(1) 
                   matrix `R' = (1, 0, 0, 0 \ 0, `infl'^2, 0, 0 \ 0, 0, `infl'^2, 0 \ 0, 0, 0, `infl'^2)
                   matrix `phit' = `infl'
                   matrix `cc' = (`phit' \ 0 \ 0 \ 0)   
				   matrix `c' = (`phit' \ 0 \ 0 \ 0 \ 0 \ 0 )  
                   matrix `x' = (1, 0, 0, 0)
                   scalar `smean' = `infl'
                   scalar `smeanx' = 0
                   matrix `perr1' = 0
                   matrix `perr2' = 0
                   matrix `perr3' = 0
                   matrix `perr4' = 0
                }
                
				//this one modified from inflation expectations code 
                else if $gain == 9 {                          // alt. Seasonal AR(1) 
                   matrix `R' = (1, 0 \ 0, `infl'^2)
                   matrix `phit' = `infl'
                   matrix `cc' = (`phit' \ 0)     // dispersion in fitted exp looks identical with 0, 0.99 as starting value
                   matrix `x' = (1,0)  
                   scalar `smean' = `infl'
                   scalar `smeanx' = 0
                }
                
                else if $gain == 10 { 
                   matrix `R' = 1    // (1, 0 \ 0, 1)
                   matrix `phit' = `infl'
                   matrix `cc' = 0     
                   matrix `x' = 0
                   scalar `smean' = 0
                   scalar `smeanx' = 0
                   matrix `perr1' = 0
                   matrix `perr2' = 0
                   matrix `perr3' = 0
                   matrix `perr4' = 0
                }
				
				if ($gain ~= 8) {
                   matrix `c' = `cc'
				} 
                
               
                // also tried MA(1): did not explain well
                   
             }
             
             
             //updating of parameter estimates 
             
             else  {
             
                if $qtr == 0 {
                   scalar `linfl' = $pred[`yri'-4,1]
                   scalar `llinfl' = $pred[`yri'-8,1]
                   scalar `dlinfl' = `linfl'-$pred[`yri'-8,1]
                   scalar `dinfl' = `infl'-$pred[`yri'-4,1]
                }
                else if $qtr == 1 {
                   scalar `linfl' = $pred[`yri'-1,1]
                   scalar `llinfl' = $pred[`yri'-2,1]
				   scalar `l3infl' = $pred[`yri'-3,1]
                   scalar `l4infl' = $pred[`yri'-4,1]
				   scalar `l5infl' = $pred[`yri'-5,1]
                   scalar `dlinfl' = `linfl'-$pred[`yri'-2,1]
                   scalar `dinfl' = `infl'-$pred[`yri'-1,1]
                   scalar `lao' = ($pred[`yri'-1,1]+$pred[`yri'-2,1]+$pred[`yri'-3,1]+$pred[`yri'-4,1])/4
                   scalar `ao' = ($pred[`yri',1]+$pred[`yri'-1,1]+$pred[`yri'-2,1]+$pred[`yri'-3,1])/4
                   scalar `wlinfl4' = (4*$pred[`yri'-1,1]+3*$pred[`yri'-2,1]+2*$pred[`yri'-3,1]+$pred[`yri'-4,1])/10
                   scalar `winfl4' = (4*$pred[`yri',1]+3*$pred[`yri'-1,1]+2*$pred[`yri'-2,1]+$pred[`yri'-3,1])/10
                   
                   
                }
                
                if $gain == 3 {
                   matrix `x' = 1
                }
                else if ($gain == 4 | $gain == 5) {
                   matrix `x' = (1,`linfl') 
                }
                else if $gain == 6 {
                   if $qtr == 0 { 
                      matrix `x' = (1,`perr4')
                   }   
                   else {
                      matrix `x' = (1,`perr1')
                   }   
                }
                else if $gain == 7 { 
                   if $qtr == 0 { 
                      matrix `x' = (1, `linfl', `perr4')
                   }   
                   else {
                      matrix `x' = (1, `linfl', `perr1')
                   }   
                }
                else if $gain == 8 { 
                    if $qtr == 0 { 
                      matrix `x' = (1, `linfl', `l4infl', `l5infl')   
                   }   
                   else {
                      matrix `x' = (1, `linfl', `l4infl', `l5infl')
                   }
                }
                else if $gain == 9 { 
                      matrix `x' = (1,`l4infl')   
                }
                else if $gain == 10 { 
                   if $qtr == 0 { 
                      matrix `x' = (1, `lao')
                   }   
                   else {
                      matrix `x' = ( `wlinfl4')
                   }
                }
                
                if $gain ~= 6 {
                   scalar `y' = `infl'
                }
                else {
                   scalar `y' = `dinfl'
                }
                
                if ($gain > 5 & $gain ~= 9) {
                   matrix `perr4' = `perr3'
                   matrix `perr3' = `perr2'
                   matrix `perr2' = `perr1'
                }
                
                //if ($gain ~= 6 ) {
                   matrix `perr1' = `y'-`cc''*`x''
                //}   
                //else {
                //   matrix `perr1' = `y'-`cc'[2,1]*`x'[1,2] 
                //}
                //else { 
                //   matrix `perr1' = `infl'-`cc''*(1 \ `linfl' \ `perr1')   // this presumes RW interpretation
                   // matrix `perr1' = `dinfl'-`cc'[2,1]*`x'[1,3] 
                //}  
                   
                
                //recursive updating step
                //if $gain ~= 10 {
                  if $gain ~= 5 {
                     matrix `R' = `R' + `g'*(`x''*`x'-`R')      
                  }
                  matrix `cc' = `cc' + `g'*invsym(`R')*`x''*`perr1'   
				  
				 // matlist `R' 
				 // matlist `cc' 
				  
                //}
                //else {
                //   matrix `R' = `R' + `gcoh'*(`x''*`x'-`R')     
                //   matrix `cc' = `cc' + `gcoh'*invsym(`R')*`x''*(`y'-`cc''*`x'')
                //}
                
                //parameter constraints
                if ($gain ~= 8) {
                   matrix `c' = `cc'
				} 
                if ($gain == 6) {
                   matrix `c'[2,1] = max(`c'[2,1],-0.999)
                   matrix `c'[2,1] = min(`c'[2,1],0)
                }
                if $gain == 7 {
                   matrix `c'[2,1] = `c'[2,1]-max(`cc'[2,1]-1,0)   // important to do manipulations on `c', not `cc'
                   matrix `c'[2,1] = `c'[2,1]-min(`cc'[2,1],0)
                   matrix `c'[1,1] = `c'[1,1] + max(`cc'[2,1]-1,0)*`smeanx' + min(`cc'[2,1],0)*`smeanx'
                   matrix `c'[3,1] = max(`c'[3,1],-0.999)
                   matrix `c'[3,1] = min(`c'[3,1],0.999)
                }
				if ($gain == 8) {
                   matrix `c'[1,1] = `cc'[1,1]
                   matrix `c'[2,1] = `cc'[2,1]
				   matrix `c'[3,1] = 0
                   matrix `c'[4,1] = 0
				   matrix `c'[5,1] = `cc'[3,1]
                   matrix `c'[6,1] = `cc'[4,1]	   
                }
                          
                scalar `smean' = `smean' + `g'*(`infl'-`smean') 
                scalar `smeanx' = `smeanx' + `g'*(`linfl'-`smeanx')  
                    
                if $gain == 3 {
                   matrix `xcurr' = 1 
                }
                else if ($gain == 4 | $gain == 5) {
                   matrix `xcurr' = (1,`infl') 
                }
                else if $gain == 6 {
                  matrix `xcurr' = (1,`perr1')
                }
                else if $gain == 7 {
                  matrix `xcurr' = (1, `infl', `perr1')
                }
                else if $gain == 8 {
                  matrix `xcurr' = (1, `infl', `linfl', `llinfl', `l3infl', `l4infl')
                }
                else if $gain == 9 {
                  matrix `xcurrlong' = (1, `infl', `linfl', `llinfl', `l3infl')
				  matrix `xcurr' = (1, `l3infl')  
                }
                else if $gain == 10 {
                  matrix `xcurr' = ( `winfl4')
                }
				
				if ($qtr == 1 )  {
                   local hf = 4*$hor-1
                }
                else {
                   local hf = $hor-1
                }
                  
                matrix `phit' = `c''*`xcurr''
				
                
                if (($gain == 4 | $gain == 5) & `hf' >= 1 ) {
                   matrix `xf' = (1, `phit')
                   forvalues h = 1/`hf' {
                      matrix `pf' = `c''*`xf''
                      matrix `phit' = `phit' + `pf'   
                      matrix `xf' = (1, `pf')
                   }
                   matrix `phit' = `phit'/(`hf'+1)    
                }       
                else if $gain == 7 {                
                   matrix `xf' = (1, `phit', 0)
                   forvalues h = 1/`hf' {
                      matrix `pf' = `c''*`xf''
                      matrix `phit' = `phit' + `pf'
                      matrix `xf' = (1, `pf', 0)
                   }
                   matrix `phit' = `phit'/(`hf'+1)
                } 
                else if $gain == 6 { 
                   matrix `phit' = `phit' + 3*`c'[1,1]
                }   
                else if ($gain == 8 & `hf' >= 1) {
                   matrix `xf' = (1, `phit', `xcurr'[1,2], `xcurr'[1,3], `xcurr'[1,4], `xcurr'[1,5])
                   forvalues h = 1/`hf' {
                      matrix `pf' = `c''*`xf''         // c has some zero entries and selects only relevant lags
                      matrix `phit' = `phit' + `pf'
                      matrix `xf' = (1, `pf', `xf'[1,2], `xf'[1,3], `xf'[1,4], `xf'[1,5] )
					  //matlist `pf'
                   }
                   matrix `phit' = `phit'/(`hf'+1) 
                }
                else if $gain == 9  {
				   matrix `xf' = (1, `xcurrlong'[1,4])
                   forvalues h = 1/`hf' {
                      matrix `pf' = `c''*`xf''         
                      matrix `phit' = `phit' + `pf'
                      matrix `xf' = (1, `xcurrlong'[1,4-`h'] )
                   }
                   matrix `phit' = `phit'/(`hf'+1) 
                }
                //if $gain == 10 {
                //   scalar `ac' = (-1+sqrt(1+4*`c'[2,1]))/2
                //   matrix `phit' = (`phit'-`c'[3,1]*`xcurr'[1,2])/(1-`c'[3,1])
                //   matrix `xf' = (1, `phit')
                //   forvalues h = 1/`hf' {
                //      matrix `pf' = (`c'[1,1]/(1-`c'[3,1]), `ac'^4)*`xf''
                //      matrix `phit' = `phit' + `pf'   
                //      matrix `xf' = (1, `pf')
                //   }
                //   matrix `phit' = (`phit'/(`hf'+1))*(1-`c'[3,1]) + `c'[3,1]*`xcurr'[1,3] 
                //}   
               
             }
                 
          if ($gain < 6 | $gain > 6) {
             matrix inflfullexp[`yri',1] = `phit'
          }    
          else {
             matrix inflfullexp[`yri',1] = `phit'+`infl'
          }
             
          if ($gain == 4 | $gain == 5 | $gain == 6 | $gain == 7 | $gain == 10)  {
             matrix acfull[`yri',1] = `c'[2,1] 
             if ($gain == 7 | $gain == 10) {
                matrix mafull[`yri',1] = `c'[3,1] 
             }
          } 
      }
   }
       
end 


*------------------------------------------------------------------------------*
* Calculate Inflation Expectations
*------------------------------------------------------------------------------*
* U.S. Stock Markets 1871-Present and CAPE Ratio
* http://www.econ.yale.edu/~shiller/data.htm

* Copy-paste part of the "Data" sheet from "ie_data.xls":
* Date	     P	   D	 E	  CPI	       Fraction	    R	       PE10
* 1871.01	4.44  0.26  0.4  12.46406116  1871.041667  5.32	        -99
* 1871.02	4.5	  0.26	0.4  12.84464132  1871.125	   5.323333333	-99
* 1871.03	4.61  0.26	0.4  13.0349719   1871.208333  5.326666667  -99
* 1871.04	4.74  0.26	0.4	 12.55922645  1871.291667  5.33	        -99
* ...       ...   ...   ...  ...          ...          ...          ...

insheet using "../Data/Shillerdata_monthly.csv", clear names

gen yyyy = floor(date)
keep if yyyy < 2020 
gen month = ceil(100*(date-yyyy))
gen qtr = floor((month+2)/3) 
replace cpi = 0 if (month ~= 3 & month ~= 6 & month ~= 9 & month ~= 12)
collapse (sum) cpi, by(yyyy qtr) 
gen qtrinfl = log(cpi/cpi[_n-1])

global datast = 1871 
keep if yyyy > $datast  // important to have this before learning algo called
gen yyyyqtr = yyyy*100+qtr

mkmat yyyyqtr
mkmat qtrinfl

local m = rowsof(qtrinfl)
local rowyrs ""

forvalues i=1/`m'  {
   local addyr = yyyyqtr[`i',1]
   local rowyrs "`rowyrs' `addyr' "
   }
matrix rownames qtrinfl = `rowyrs'

global yrst 1872   
global qtr 1 
global hor 0.25   // horizon in years 
qui gen iexploc = (yyyy-$yrst)*4+qtr-1  // yyyy and q should be the date (year, qtr) of the observation 
                                     // the algorithm then calculates the learning-from-experience forecast using  
                                     // data up to the previous quarter 
 
* need just one-quarter forecasts: 
global gain 8
global pred "qtrinfl"
constgain 0.018 
qui gen expinfl = inflfullexp[iexploc,1]  
 
global gain 4
global pred "qtrinfl"
constgain 0.018 
qui gen expinflnoseas = inflfullexp[iexploc,1]  
 
keep if yyyy > 1925 

*------------------------------------------------------------------------------* 
* Output Inflation Expectations
*------------------------------------------------------------------------------*
keep yyyy qtr expinfl expinflnoseas

export delimited using inflexp, replace


